R Notebook

Shiny App Experiment Lab

Libraries

library(here)
library(dplyr)
library(arrow)
library(tidyr)
library(geojsonio)

Reading data

census_dataset <- open_dataset(here("data", "processed", "parquet_data_coords"))
census_dataset

FileSystemDataset with 33 Parquet files
NOM_MUN: string
NOM_LOC: string
LONGITUD: string
LATITUD: string
POBTOT: double
REL_H_M: string
POB0_14: string
POB15_64: string
POB65_MAS: string
P_0A4: string
P_0A4_F: string
P_0A4_M: string
P_5A9: string
P_5A9_F: string
P_5A9_M: string
P_10A14: string
P_10A14_F: string
P_10A14_M: string
P_15A19: string
P_15A19_F: string
P_15A19_M: string
P_20A24: string
P_20A24_F: string
P_20A24_M: string
P_25A29: string
P_25A29_F: string
P_25A29_M: string
P_30A34: string
P_30A34_F: string
P_30A34_M: string
P_35A39: string
P_35A39_F: string
P_35A39_M: string
P_40A44: string
P_40A44_F: string
P_40A44_M: string
P_45A49: string
P_45A49_F: string
P_45A49_M: string
P_50A54: string
P_50A54_F: string
P_50A54_M: string
P_55A59: string
P_55A59_F: string
P_55A59_M: string
P_60A64: string
P_60A64_F: string
P_60A64_M: string
P_65A69: string
P_65A69_F: string
P_65A69_M: string
P_70A74: string
P_70A74_F: string
P_70A74_M: string
P_75A79: string
P_75A79_F: string
P_75A79_M: string
P_80A84: string
P_80A84_F: string
P_80A84_M: string
P_85YMAS: string
P_85YMAS_F: string
P_85YMAS_M: string
PROM_HNV: string
PNACENT: string
PNACENT_F: string
PNACENT_M: string
PNACOE: string
PNACOE_F: string
PNACOE_M: string
longitude_decimal: double
latitude_decimal: double
NOM_ENT: string

See $metadata for additional Schema metadata

Reading specific data

pueb_norm <- census_dataset |>
    filter(NOM_ENT=="Puebla") |> 
    collect()

Warning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$r

pueb_norm

unique(pueb_norm$NOM_MUN)

  [1] "Total de la entidad Puebla"       "Acajete"                          "Acateno"                          "Acatlán"                         
  [5] "Acatzingo"                        "Acteopan"                         "Ahuacatlán"                       "Ahuatlán"                        
  [9] "Ahuazotepec"                      "Ahuehuetitla"                     "Ajalpan"                          "Albino Zertuche"                 
 [13] "Aljojuca"                         "Altepexi"                         "Amixtlán"                         "Amozoc"                          
 [17] "Aquixtla"                         "Atempan"                          "Atexcal"                          "Atlixco"                         
 [21] "Atoyatempan"                      "Atzala"                           "Atzitzihuacán"                    "Atzitzintla"                     
 [25] "Axutla"                           "Ayotoxco de Guerrero"             "Calpan"                           "Caltepec"                        
 [29] "Camocuautla"                      "Caxhuacan"                        "Coatepec"                         "Coatzingo"                       
 [33] "Cohetzala"                        "Cohuecan"                         "Coronango"                        "Coxcatlán"                       
 [37] "Coyomeapan"                       "Coyotepec"                        "Cuapiaxtla de Madero"             "Cuautempan"                      
 [41] "Cuautinchán"                      "Cuautlancingo"                    "Cuayuca de Andrade"               "Cuetzalan del Progreso"          
 [45] "Cuyoaco"                          "Chalchicomula de Sesma"           "Chapulco"                         "Chiautla"                        
 [49] "Chiautzingo"                      "Chiconcuautla"                    "Chichiquila"                      "Chietla"                         
 [53] "Chigmecatitlán"                   "Chignahuapan"                     "Chignautla"                       "Chila"                           
 [57] "Chila de la Sal"                  "Honey"                            "Chilchotla"                       "Chinantla"                       
 [61] "Domingo Arenas"                   "Eloxochitlán"                     "Epatlán"                          "Esperanza"                       
 [65] "Francisco Z. Mena"                "General Felipe Ángeles"           "Guadalupe"                        "Guadalupe Victoria"              
 [69] "Hermenegildo Galeana"             "Huaquechula"                      "Huatlatlauca"                     "Huauchinango"                    
 [73] "Huehuetla"                        "Huehuetlán el Chico"              "Huejotzingo"                      "Hueyapan"                        
 [77] "Hueytamalco"                      "Hueytlalpan"                      "Huitzilan de Serdán"              "Huitziltepec"                    
 [81] "Atlequizayan"                     "Ixcamilpa de Guerrero"            "Ixcaquixtla"                      "Ixtacamaxtitlán"                 
 [85] "Ixtepec"                          "Izúcar de Matamoros"              "Jalpan"                           "Jolalpan"                        
 [89] "Jonotla"                          "Jopala"                           "Juan C. Bonilla"                  "Juan Galindo"                    
 [93] "Juan N. Méndez"                   "Lafragua"                         "Libres"                           "La Magdalena Tlatlauquitepec"    
 [97] "Mazapiltepec de Juárez"           "Mixtla"                           "Molcaxac"                         "Cañada Morelos"                  
[101] "Naupan"                           "Nauzontla"                        "Nealtican"                        "Nicolás Bravo"                   
[105] "Nopalucan"                        "Ocotepec"                         "Ocoyucan"                         "Olintla"                         
[109] "Oriental"                         "Pahuatlán"                        "Palmar de Bravo"                  "Pantepec"                        
[113] "Petlalcingo"                      "Piaxtla"                          "Puebla"                           "Quecholac"                       
[117] "Quimixtlán"                       "Rafael Lara Grajales"             "Los Reyes de Juárez"              "San Andrés Cholula"              
[121] "San Antonio Cañada"               "San Diego la Mesa Tochimiltzingo" "San Felipe Teotlalcingo"          "San Felipe Tepatlán"             
[125] "San Gabriel Chilac"               "San Gregorio Atzompa"             "San Jerónimo Tecuanipan"          "San Jerónimo Xayacatlán"         
[129] "San José Chiapa"                  "San José Miahuatlán"              "San Juan Atenco"                  "San Juan Atzompa"                
[133] "San Martín Texmelucan"            "San Martín Totoltepec"            "San Matías Tlalancaleca"          "San Miguel Ixitlán"              
[137] "San Miguel Xoxtla"                "San Nicolás Buenos Aires"         "San Nicolás de los Ranchos"       "San Pablo Anicano"               
[141] "San Pedro Cholula"                "San Pedro Yeloixtlahuaca"         "San Salvador el Seco"             "San Salvador el Verde"           
[145] "San Salvador Huixcolotla"         "San Sebastián Tlacotepec"         "Santa Catarina Tlaltempan"        "Santa Inés Ahuatempan"           
[149] "Santa Isabel Cholula"             "Santiago Miahuatlán"              "Huehuetlán el Grande"             "Santo Tomás Hueyotlipan"         
[153] "Soltepec"                         "Tecali de Herrera"                "Tecamachalco"                     "Tecomatlán"                      
[157] "Tehuacán"                         "Tehuitzingo"                      "Tenampulco"                       "Teopantlán"                      
[161] "Teotlalco"                        "Tepanco de López"                 "Tepango de Rodríguez"             "Tepatlaxco de Hidalgo"           
[165] "Tepeaca"                          "Tepemaxalco"                      "Tepeojuma"                        "Tepetzintla"                     
[169] "Tepexco"                          "Tepexi de Rodríguez"              "Tepeyahualco"                     "Tepeyahualco de Cuauhtémoc"      
[173] "Tetela de Ocampo"                 "Teteles de Avila Castillo"        "Teziutlán"                        "Tianguismanalco"                 
[177] "Tilapa"                           "Tlacotepec de Benito Juárez"      "Tlacuilotepec"                    "Tlachichuca"                     
[181] "Tlahuapan"                        "Tlaltenango"                      "Tlanepantla"                      "Tlaola"                          
[185] "Tlapacoya"                        "Tlapanalá"                        "Tlatlauquitepec"                  "Tlaxco"                          
[189] "Tochimilco"                       "Tochtepec"                        "Totoltepec de Guerrero"           "Tulcingo"                        
[193] "Tuzamapan de Galeana"             "Tzicatlacoyan"                    "Venustiano Carranza"              "Vicente Guerrero"                
[197] "Xayacatlán de Bravo"              "Xicotepec"                        "Xicotlán"                         "Xiutetelco"                      
[201] "Xochiapulco"                      "Xochiltepec"                      "Xochitlán de Vicente Suárez"      "Xochitlán Todos Santos"          
[205] "Yaonáhuac"                        "Yehualtepec"                      "Zacapala"                         "Zacapoaxtla"                     
[209] "Zacatlán"                         "Zapotitlán"                       "Zapotitlán de Méndez"             "Zaragoza"                        
[213] "Zautla"                           "Zihuateutla"                      "Zinacatepec"                      "Zongozotla"                      
[217] "Zoquiapan"                        "Zoquitlán"                       

Example

extract_coordinates <- function(data, municipality, locality) {
  selected_location <- data |> 
    filter(NOM_MUN == municipality, NOM_LOC == locality)
  
  coordinates <- tibble(
    long = selected_location$longitude_decimal,
    lat = selected_location$latitude_decimal
  )
  
  return(coordinates)
}


municipality <- "Acajete"
locality <- "Santa Isabel Tepetzala"

red_point <- extract_coordinates(pueb_norm, municipality, locality) |> 
  slice(1)

# red_point <- data.frame(long = -98.2035, lat = 19.0414)
geojson_file <- geojson_read("../data/processed/mexico.geojson",  what = "sp")

filtered_geojson <- geojson_file |> 
  filter(name == "Puebla")

ggplot() +
  geom_polygon(data = filtered_geojson,
               aes(x = long, y = lat, group = group),
               fill = "lightgray", color = "white") +
  geom_point(data = red_point, aes(x = long, y = lat), color = "red", size = 3) +
  theme_void() +
  coord_map()

Warning: `fortify(<SpatialPolygonsDataFrame>)` was deprecated in ggplot2 3.4.4.
Please migrate to sf.Regions defined for each Polygons

Population graph

pueb_norm

Getting column names

column_names <- names(pueb_norm)
column_names

 [1] "NOM_MUN"           "NOM_LOC"           "LONGITUD"          "LATITUD"           "POBTOT"            "REL_H_M"           "POB0_14"          
 [8] "POB15_64"          "POB65_MAS"         "P_0A4"             "P_0A4_F"           "P_0A4_M"           "P_5A9"             "P_5A9_F"          
[15] "P_5A9_M"           "P_10A14"           "P_10A14_F"         "P_10A14_M"         "P_15A19"           "P_15A19_F"         "P_15A19_M"        
[22] "P_20A24"           "P_20A24_F"         "P_20A24_M"         "P_25A29"           "P_25A29_F"         "P_25A29_M"         "P_30A34"          
[29] "P_30A34_F"         "P_30A34_M"         "P_35A39"           "P_35A39_F"         "P_35A39_M"         "P_40A44"           "P_40A44_F"        
[36] "P_40A44_M"         "P_45A49"           "P_45A49_F"         "P_45A49_M"         "P_50A54"           "P_50A54_F"         "P_50A54_M"        
[43] "P_55A59"           "P_55A59_F"         "P_55A59_M"         "P_60A64"           "P_60A64_F"         "P_60A64_M"         "P_65A69"          
[50] "P_65A69_F"         "P_65A69_M"         "P_70A74"           "P_70A74_F"         "P_70A74_M"         "P_75A79"           "P_75A79_F"        
[57] "P_75A79_M"         "P_80A84"           "P_80A84_F"         "P_80A84_M"         "P_85YMAS"          "P_85YMAS_F"        "P_85YMAS_M"       
[64] "PROM_HNV"          "PNACENT"           "PNACENT_F"         "PNACENT_M"         "PNACOE"            "PNACOE_F"          "PNACOE_M"         
[71] "longitude_decimal" "latitude_decimal"  "NOM_ENT"          

Getting column names that have to do with population age cohorts (Masculine/Feminine)

matching_columns <- grep("^P_.*[MF]$", column_names, value = TRUE)
matching_columns

 [1] "P_0A4_F"    "P_0A4_M"    "P_5A9_F"    "P_5A9_M"    "P_10A14_F"  "P_10A14_M"  "P_15A19_F"  "P_15A19_M"  "P_20A24_F"  "P_20A24_M"  "P_25A29_F" 
[12] "P_25A29_M"  "P_30A34_F"  "P_30A34_M"  "P_35A39_F"  "P_35A39_M"  "P_40A44_F"  "P_40A44_M"  "P_45A49_F"  "P_45A49_M"  "P_50A54_F"  "P_50A54_M" 
[23] "P_55A59_F"  "P_55A59_M"  "P_60A64_F"  "P_60A64_M"  "P_65A69_F"  "P_65A69_M"  "P_70A74_F"  "P_70A74_M"  "P_75A79_F"  "P_75A79_M"  "P_80A84_F" 
[34] "P_80A84_M"  "P_85YMAS_F" "P_85YMAS_M"

Separate by sex

ending_in_M <- character(0)
ending_in_F <- character(0)

for (col_name in matching_columns) {
  if (endsWith(col_name, "M")) {
    ending_in_M <- c(ending_in_M, col_name)
  } else if (endsWith(col_name, "F")) {
    ending_in_F <- c(ending_in_F, col_name)
  }
}

print("Column names ending in M:")

[1] "Column names ending in M:"

print(ending_in_M)

 [1] "P_0A4_M"    "P_5A9_M"    "P_10A14_M"  "P_15A19_M"  "P_20A24_M"  "P_25A29_M"  "P_30A34_M"  "P_35A39_M"  "P_40A44_M"  "P_45A49_M"  "P_50A54_M" 
[12] "P_55A59_M"  "P_60A64_M"  "P_65A69_M"  "P_70A74_M"  "P_75A79_M"  "P_80A84_M"  "P_85YMAS_M"

print("Column names ending in F:")

[1] "Column names ending in F:"

print(ending_in_F)

 [1] "P_0A4_F"    "P_5A9_F"    "P_10A14_F"  "P_15A19_F"  "P_20A24_F"  "P_25A29_F"  "P_30A34_F"  "P_35A39_F"  "P_40A44_F"  "P_45A49_F"  "P_50A54_F" 
[12] "P_55A59_F"  "P_60A64_F"  "P_65A69_F"  "P_70A74_F"  "P_75A79_F"  "P_80A84_F"  "P_85YMAS_F"

cohort_names_m <- c("P_0A4_M",
                    "P_5A9_M",
                    "P_10A14_M",
                    "P_15A19_M",
                    "P_20A24_M",
                    "P_25A29_M",
                    "P_30A34_M",
                    "P_35A39_M",
                    "P_40A44_M",
                    "P_45A49_M",
                    "P_50A54_M",
                    "P_55A59_M",
                    "P_60A64_M",
                    "P_65A69_M",
                    "P_70A74_M",
                    "P_75A79_M",
                    "P_80A84_M",
                    "P_85YMAS_M")

cohort_names_f <- c("P_0A4_F",
                    "P_5A9_F",
                    "P_10A14_F",
                    "P_15A19_F",
                    "P_20A24_F",
                    "P_25A29_F",
                    "P_30A34_F",
                    "P_35A39_F",
                    "P_40A44_F",
                    "P_45A49_F",
                    "P_50A54_F",
                    "P_55A59_F",
                    "P_60A64_F",
                    "P_65A69_F", 
                    "P_70A74_F",
                    "P_75A79_F",
                    "P_80A84_F", 
                    "P_85YMAS_F")



municipality <- "Acajete"
locality <- "San Javier"

pueb_norm_filt <- pueb_norm |> 
    filter(NOM_MUN == municipality, NOM_LOC == locality)

cohort_counts_m <- as.numeric(pueb_norm_filt[1,cohort_names_m])
cohort_counts_f <- as.numeric(pueb_norm_filt[1,cohort_names_f])

data <- tibble(
  Cohort = c(cohort_names_m, cohort_names_f),
  Count = c(cohort_counts_m, cohort_counts_f),
  Sex = rep(c("Male", "Female"), each = length(cohort_names_m))
)

# Plotting population pyramid
ggplot(data, aes(x = reorder(Cohort, -Count), y = Count, fill = Sex)) +
  geom_bar(stat = "identity", position = "identity") +
  scale_fill_manual(values = c("blue", "pink")) +  
  coord_flip() +  
  labs(title = "Population Pyramid",
       x = "Population Count",
       y = "Age Cohort",
       fill = "Sex") +
  theme_minimal()  

new_ages <- c("0-4",
                 "5-9",
                 "10-14",
                 "15-19",
                 "20-24",
                 "25-29", 
                 "30-34", 
                 "35-39", 
                 "40-44", 
                 "45-49",
                 "50-54",
                 "55-59",
                 "60-64", 
                 "65-69",
                 "70-74", 
                 "75-79", 
                 "80-84", 
                 "85+")

data <- tibble(
  Age = paste0(new_ages),
  Male = sample(200:1000, length(cohort_names_m), replace = TRUE),
  Female = sample(200:1000, length(cohort_names_f), replace = TRUE)
)

data_long <- pivot_longer(
  data, 
  cols = c(Male, Female), 
  names_to = "Sex", 
  values_to = "Population"
)

basic_plot <- ggplot(data_long, aes(x = Age, y = ifelse(Sex == "Male", -Population, Population), fill = Sex)) +
  geom_bar(stat = "identity") +
  scale_y_continuous(labels = abs, limits = max(data_long$Population) * c(-1, 1)) +
  coord_flip() +
  theme_minimal() +
  labs(x = "Age", y = "Population", fill = "Sex", title = "Population Pyramid")

basic_plot

Complete pipeline

census_dataset <- open_dataset(here("data", "processed", "parquet_data_coords"))


pueb_norm <- census_dataset |>
    filter(NOM_ENT=="Puebla") |> 
    collect()

Warning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$r

municipality <- "Acajete"
locality <- "San Javier"

pueb_norm_filt <- pueb_norm |> 
    filter(NOM_MUN == municipality, NOM_LOC == locality)


cohort_names_m <- c("P_0A4_M",
                    "P_5A9_M",
                    "P_10A14_M",
                    "P_15A19_M",
                    "P_20A24_M",
                    "P_25A29_M",
                    "P_30A34_M",
                    "P_35A39_M",
                    "P_40A44_M",
                    "P_45A49_M",
                    "P_50A54_M",
                    "P_55A59_M",
                    "P_60A64_M",
                    "P_65A69_M",
                    "P_70A74_M",
                    "P_75A79_M",
                    "P_80A84_M",
                    "P_85YMAS_M")

cohort_names_f <- c("P_0A4_F",
                    "P_5A9_F",
                    "P_10A14_F",
                    "P_15A19_F",
                    "P_20A24_F",
                    "P_25A29_F",
                    "P_30A34_F",
                    "P_35A39_F",
                    "P_40A44_F",
                    "P_45A49_F",
                    "P_50A54_F",
                    "P_55A59_F",
                    "P_60A64_F",
                    "P_65A69_F", 
                    "P_70A74_F",
                    "P_75A79_F",
                    "P_80A84_F", 
                    "P_85YMAS_F")



new_ages <- c("0-4",
                 "5-9",
                 "10-14",
                 "15-19",
                 "20-24",
                 "25-29", 
                 "30-34", 
                 "35-39", 
                 "40-44", 
                 "45-49",
                 "50-54",
                 "55-59",
                 "60-64", 
                 "65-69",
                 "70-74", 
                 "75-79", 
                 "80-84", 
                 "85+")


data <- tibble(
  Age = paste0(new_ages),
  Male = as.numeric(pueb_norm_filt[1,cohort_names_m]),
  Female = as.numeric(pueb_norm_filt[1,cohort_names_f])
)

data_long <- pivot_longer(
  data, 
  cols = c(Male, Female), 
  names_to = "Sex", 
  values_to = "Population"
)
basic_plot <- ggplot(data_long, aes(x = Age, y = ifelse(Sex == "Male", -Population, Population), fill = Sex)) +
  geom_bar(stat = "identity") +
  scale_y_continuous(labels = abs, limits = max(data_long$Population) * c(-1, 1)) +
  coord_flip() +
  theme_minimal() +
  labs(x = "Age", y = "Population", fill = "Sex", title = "Population Pyramid")

basic_plot

Card

card <- census_dataset |>
    filter(
      NOM_ENT == "Puebla",
      NOM_MUN == "Acateno",
      NOM_LOC == "Santa Andrea"
    ) |>
  collect()

Warning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$r

    
    total_population <- as.numeric(card[1, c("POBTOT")], na.rm = TRUE)
    paste("Total Population:", total_population)

[1] "Total Population: 1"

Pie

tot <- census_dataset |>
    filter(NOM_ENT=="Total nacional") |> 
    collect()

Warning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$rWarning: Invalid metadata$r

tot

# origin <- census_dataset |>  
#      filter(
#       NOM_ENT == "Puebla",
#       NOM_MUN == "Acateno",
#       NOM_LOC == "Santa Andrea"
#     ) |>
#   collect()
    
tot

    
    # Extract birth data
    birth_local <- as.numeric(tot[1, "PNACENT"])
    birth_another <- as.numeric(tot[1, "PNACOE"])
    
    # Debugging output
    print(paste("Birth Local:", birth_local))

[1] "Birth Local: 102724322"

    print(paste("Birth Another:", birth_another))

[1] "Birth Another: 21611963"

    
    # Create ratio dataframe
    ratio_df <- tibble(
      Category = c("Local", "Other"),
      Ratio = c(birth_local, birth_another)
    )
    
    # Calculate percentages
    ratio_df$Percentage <- ratio_df$Ratio / sum(ratio_df$Ratio) * 100
    
    # Plot the pie chart
    gg <- ggplot(ratio_df, aes(x = "", y = Ratio, fill = Category)) +
      geom_bar(stat = "identity", width = 1) +
      coord_polar(theta = "y") +
      theme_void() +
      theme(legend.position = "bottom") +
      scale_fill_manual(values = c("#00BFC4", "#F8766D")) +
      geom_text(aes(label = paste0(round(Percentage), "%")), 
                position = position_stack(vjust = 0.5),
                size = 5, color = "white", fontface = "bold")
    
    gg

NA

Read CSV from URL

test_csv <- read.csv("https://raw.github.ubc.ca/MDS-2023-24/DSCI_532_individual-assignment_marcony1/master/data/processed/data_coords.csv?token=GHSAT0AAAAAAAAACLO5WK36WJ6S74E3NU4CZRJJJGA")

test_csv

NA

filtered_data <- filter(test_csv, NOM_ENT == "Puebla")
filtered_data

Download with curl

library(curl)

# Define the URL of the CSV file
url <- "https://raw.github.ubc.ca/MDS-2023-24/DSCI_532_individual-assignment_marcony1/master/data/processed/data_coords.csv?token=GHSAT0AAAAAAAAACLO5WK36WJ6S74E3NU4CZRJJJGA"

response <- curl::curl_fetch_memory(url)

if (response$status_code == 200) {
  csv_content <- rawToChar(response$content)
    test_csv <- read.csv(text = csv_content)
print(test_csv)
}

library(RCurl)
x <- getURL("https://raw.github.ubc.ca/MDS-2023-24/DSCI_532_individual-assignment_marcony1/master/data/processed/data_coords.csv?token=GHSAT0AAAAAAAAACLO5WK36WJ6S74E3NU4CZRJJJGA")
y <- read.csv(text = x)
y

test_2 <- arrow::read_parquet("https://github.ubc.ca/MDS-2023-24/DSCI_532_individual-assignment_marcony1/tree/master/data/processed/parquet_data_coords.parquet")

Error: Invalid: Parquet magic bytes not found in footer. Either the file is corrupted or this is not a parquet file.